{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib import learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Regression For the Boston Housing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import datasets, metrics, preprocessing\n",
    "boston = datasets.load_boston()\n",
    "x_data = preprocessing.StandardScaler().fit_transform(boston.data)\n",
    "y_data = boston.target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Native TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float64,shape=(None,13))\n",
    "y_true = tf.placeholder(tf.float64,shape=(None))\n",
    "\n",
    "with tf.name_scope('inference') as scope:\n",
    "    w = tf.Variable(tf.zeros([1,13],dtype=tf.float64,name='weights'))\n",
    "    b = tf.Variable(0,dtype=tf.float64,name='bias')\n",
    "    y_pred = tf.matmul(w,tf.transpose(x)) + b\n",
    "\n",
    "with tf.name_scope('loss') as scope:\n",
    "    loss = tf.reduce_mean(tf.square(y_true-y_pred))\n",
    "\n",
    "with tf.name_scope('train') as scope:\n",
    "    learning_rate = 0.1\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    train = optimizer.minimize(loss)\n",
    "\n",
    "# Before starting, initialize the variables.  We will 'run' this first.\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)      \n",
    "    for step in range(200):\n",
    "        sess.run(train,{x: x_data, y_true: y_data})\n",
    "        \n",
    "    MSE = sess.run(loss,{x: x_data, y_true: y_data})\n",
    "print(MSE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### contirb.learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "NUM_STEPS = 200\n",
    "MINIBATCH_SIZE = 506\n",
    "\n",
    "feature_columns = learn.infer_real_valued_columns_from_input(x_data)\n",
    "\n",
    "reg = learn.LinearRegressor(\n",
    "      feature_columns=feature_columns,\n",
    "      optimizer=tf.train.GradientDescentOptimizer(\n",
    "      learning_rate=0.1)\n",
    "      )\n",
    "\n",
    "reg.fit(x_data, boston.target, steps=NUM_STEPS, \n",
    "        batch_size=MINIBATCH_SIZE)\n",
    "\n",
    "MSE = reg.evaluate(x_data, boston.target, steps=1)\n",
    "\n",
    "print(MSE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MNIST with contrib.learn DNN classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "DATA_DIR = '/tmp/data' if not 'win32' in sys.platform else \"c:\\\\tmp\\\\data\"\n",
    "data = input_data.read_data_sets(DATA_DIR, one_hot=False)\n",
    "x_data, y_data = data.train.images,data.train.labels.astype(np.int32)\n",
    "x_test, y_test = data.test.images,data.test.labels.astype(np.int32)\n",
    "\n",
    "\n",
    "NUM_STEPS = 2000\n",
    "MINIBATCH_SIZE = 128\n",
    "\n",
    "feature_columns = learn.infer_real_valued_columns_from_input(x_data)\n",
    "\n",
    "dnn = learn.DNNClassifier(\n",
    "    feature_columns=feature_columns,\n",
    "    hidden_units=[200],\n",
    "    n_classes=10,\n",
    "    optimizer=tf.train.ProximalAdagradOptimizer(\n",
    "    learning_rate=0.2)\n",
    "    )\n",
    "\n",
    "dnn.fit(x=x_data,y=y_data, steps=NUM_STEPS,\n",
    "        batch_size=MINIBATCH_SIZE)\n",
    "\n",
    "test_acc = dnn.evaluate(x=x_test,y=y_test, steps=1)[\"accuracy\"]\n",
    "print('test accuracy: {}'.format(test_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict and plot confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap,aspect='auto')\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    \n",
    "    \n",
    "    plt.savefig('confusion_mat.png', bbox_inches='tight', format='png', dpi=300, pad_inches=0,transparent=True)\n",
    "\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import itertools\n",
    "\n",
    "y_pred = dnn.predict(x=x_test,as_iterable=False)\n",
    "class_names = ['0','1','2','3','4','5','6','7','8','9']    \n",
    "cnf_matrix = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "plot_confusion_matrix(cnf_matrix, class_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate example categorical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "N = 10000\n",
    "\n",
    "weight = np.random.randn(N)*5+70\n",
    "spec_id = np.random.randint(0,3,N)\n",
    "bias = [0.9,1,1.1]\n",
    "height = np.array([weight[i]/100 + bias[b] for i,b in enumerate(spec_id)])\n",
    "spec_name = ['Goblin','Human','ManBears']\n",
    "spec = [spec_name[s] for s in spec_id]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### plot and create data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "N = 10000\n",
    "\n",
    "weight = np.random.randn(N)*5+70\n",
    "spec_id = np.random.randint(0,3,N)\n",
    "bias = [0.9,1,1.1]\n",
    "height = np.array([weight[i]/100 + bias[b] for i,b in enumerate(spec_id)])\n",
    "spec_name = ['Goblin','Human','ManBears']\n",
    "spec = [spec_name[s] for s in spec_id]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "colors = ['r','b','g']\n",
    "f,axarr = plt.subplots(1,2,figsize = [7,3])\n",
    "ax = axarr[0]\n",
    "for ii in range(3):\n",
    "    ax.hist(height[spec_id == ii],50,color=colors[ii],alpha=0.5)\n",
    "    ax.set_xlabel('Height')\n",
    "    ax.set_ylabel('Frequency')\n",
    "    ax.set_title('Heights distribution')\n",
    "# ax.legend(['Goblins','Humans','ManBears'],loc=2, shadow=True,prop={'size':6})\n",
    "height = height + np.random.randn(N)*0.015\n",
    "ax.text(1.42,150,'Goblins')\n",
    "ax.text(1.63,210,'Humans')\n",
    "ax.text(1.85,150,'ManBears')\n",
    "\n",
    "ax.set_ylim([0,260])\n",
    "ax.set_xlim([1.38,2.05])\n",
    "\n",
    "df = pd.DataFrame({'Species':spec,'Weight':weight,'Height':height})\n",
    "\n",
    "\n",
    "ax = axarr[1]\n",
    "\n",
    "ax.plot(df['Height'],df['Weight'],'o',alpha=0.3,mfc='w',mec='b')\n",
    "ax.set_xlabel('Weight')\n",
    "ax.set_ylabel('Height')\n",
    "ax.set_title('Heights vs. Weights')\n",
    "    \n",
    "plt.tight_layout()\n",
    "plt.savefig('test.png', bbox_inches='tight', format='png', dpi=300)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Estimate weights with contrib.learn using Feature columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def input_fn(df):\n",
    "    feature_cols = {}\n",
    "    feature_cols['Weight'] = tf.constant(df['Weight'].values)\n",
    "    \n",
    "    feature_cols['Species'] =  tf.SparseTensor(\n",
    "    indices=[[i, 0] for i in range(df['Species'].size)],\n",
    "    values=df['Species'].values,\n",
    "    dense_shape=[df['Species'].size, 1])\n",
    "                    \n",
    "    labels = tf.constant(df['Height'].values)\n",
    "\n",
    "    return feature_cols, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib import layers\n",
    "Weight = layers.real_valued_column(\"Weight\")\n",
    "\n",
    "Species = layers.sparse_column_with_keys(\n",
    "    column_name=\"Species\", keys=['Goblin','Human','ManBears'])\n",
    "\n",
    "reg = learn.LinearRegressor(feature_columns=[Weight,Species])\n",
    "reg.fit(input_fn=lambda:input_fn(df), steps=50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w_w = reg.get_variable_value('linear/Weight/weight')\n",
    "print('Estimation for Weight: {}'.format(w_w))\n",
    "\n",
    "s_w = reg.get_variable_value('linear/Species/weights')\n",
    "b = reg.get_variable_value('linear/bias_weight')\n",
    "print('Estimation for Species: {}'.format(s_w + b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create custom CNN Estimator "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_fn(x, target, mode, params):\n",
    "    y_ = tf.cast(target, tf.float32)\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "    # Conv layer 1\n",
    "    conv1 = layers.convolution2d(x_image, 32, [5,5],\n",
    "                activation_fn=tf.nn.relu,\n",
    "                biases_initializer=tf.constant_initializer(0.1),\n",
    "                weights_initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "    pool1 = layers.max_pool2d(conv1, [2,2])\n",
    "\n",
    "    # Conv layer 2\n",
    "    conv2 = layers.convolution2d(pool1, 64, [5,5],\n",
    "                activation_fn=tf.nn.relu,\n",
    "                biases_initializer=tf.constant_initializer(0.1),\n",
    "                weights_initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "    pool2 = layers.max_pool2d(conv2, [2,2])\n",
    "\n",
    "    # FC layer\n",
    "    pool2_flat = tf.reshape(pool2, [-1, 7*7*64])\n",
    "    fc1 = layers.fully_connected(pool2_flat, 1024,\n",
    "              activation_fn=tf.nn.relu,\n",
    "              biases_initializer=tf.constant_initializer(0.1),\n",
    "              weights_initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "    fc1_drop = layers.dropout(fc1, keep_prob=params[\"dropout\"],\n",
    "        is_training=(mode == 'train'))\n",
    "\n",
    "    # readout layer\n",
    "    y_conv = layers.fully_connected(fc1_drop, 10, activation_fn=None)\n",
    "\n",
    "    cross_entropy = tf.reduce_mean(\n",
    "        tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y_))\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "        loss=cross_entropy,\n",
    "        global_step=tf.contrib.framework.get_global_step(),\n",
    "        learning_rate=params[\"learning_rate\"],\n",
    "        optimizer=\"Adam\")\n",
    "\n",
    "    predictions = tf.argmax(y_conv, 1)\n",
    "    return predictions, cross_entropy, train_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib import layers\n",
    "\n",
    "data = input_data.read_data_sets(DATA_DIR, one_hot=True)\n",
    "x_data, y_data = data.train.images,np.int32(data.train.labels)\n",
    "tf.cast(x_data,tf.float32)\n",
    "tf.cast(y_data,tf.float32)\n",
    "\n",
    "model_params = {\"learning_rate\": 1e-4, \"dropout\": 0.5}\n",
    "\n",
    "CNN = tf.contrib.learn.Estimator(\n",
    "    model_fn=model_fn, params=model_params)\n",
    "\n",
    "print(\"Starting training for %s steps max\" % 5000)\n",
    "CNN.fit(x=data.train.images,\n",
    "        y=data.train.labels, batch_size=50,\n",
    "        max_steps=5000)\n",
    "\n",
    "test_acc = 0\n",
    "for ii in range(5):\n",
    "    batch = data.test.next_batch(2000)\n",
    "    predictions = list(CNN.predict(batch[0], as_iterable=True))\n",
    "    test_acc = test_acc + (np.argmax(batch[1],1) == predictions).mean()\n",
    "\n",
    "print(test_acc/5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TFLearn\n",
    "### CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tflearn\n",
    "from tflearn.layers.core import input_data, dropout, fully_connected\n",
    "from tflearn.layers.conv import conv_2d, max_pool_2d\n",
    "from tflearn.layers.normalization import local_response_normalization\n",
    "from tflearn.layers.estimator import regression\n",
    "\n",
    "# Data loading and basic trasformations\n",
    "import tflearn.datasets.mnist as mnist\n",
    "X, Y, X_test, Y_test = mnist.load_data(one_hot=True)\n",
    "X = X.reshape([-1, 28, 28, 1])\n",
    "X_test = X_test.reshape([-1, 28, 28, 1])\n",
    "\n",
    "# Building the network\n",
    "CNN = input_data(shape=[None, 28, 28, 1], name='input')\n",
    "CNN = conv_2d(CNN, 32, 5, activation='relu', regularizer=\"L2\")\n",
    "CNN = max_pool_2d(CNN, 2)\n",
    "CNN = local_response_normalization(CNN)\n",
    "CNN = conv_2d(CNN, 64, 5, activation='relu', regularizer=\"L2\")\n",
    "CNN = max_pool_2d(CNN, 2)\n",
    "CNN = local_response_normalization(CNN)\n",
    "CNN = fully_connected(CNN, 1024, activation=None)\n",
    "CNN = dropout(CNN, 0.5)\n",
    "CNN = fully_connected(CNN, 10, activation='softmax')\n",
    "CNN = regression(CNN, optimizer='adam', learning_rate=0.0001,\n",
    "                     loss='categorical_crossentropy', name='target')\n",
    "\n",
    "# Training the network\n",
    "model = tflearn.DNN(CNN,tensorboard_verbose=0,tensorboard_dir = 'MNIST_tflearn_board/',\\\n",
    "                    checkpoint_path = 'MNIST_tflearn_checkpoints/checkpoint')\n",
    "model.fit({'input': X}, {'target': Y}, n_epoch=3, \n",
    "           validation_set=({'input': X_test}, {'target': Y_test}),\n",
    "           snapshot_step=1000,show_metric=True, run_id='convnet_mnist')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evaluation = model.evaluate({'input': X_test},{'target': Y_test})\n",
    "print(evaluation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred = model.predict({'input': X_test})\n",
    "print((np.argmax(testY,1)==np.argmax(pred,1)).mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tflearn.data_utils import to_categorical, pad_sequences\n",
    "from tflearn.datasets import imdb\n",
    "\n",
    "# Load data\n",
    "train, test, _ = imdb.load_data(path='imdb.pkl', n_words=10000,\n",
    "                                valid_portion=0.1)\n",
    "X_train, Y_train = train\n",
    "X_test, Y_test = test\n",
    "\n",
    "# Sequence padding and Converting labels to binary vectors\n",
    "X_train = pad_sequences(X_train, maxlen=100, value=0.)\n",
    "X_test = pad_sequences(X_test, maxlen=100, value=0.)\n",
    "Y_train = to_categorical(Y_train, nb_classes=2)\n",
    "Y_test = to_categorical(Y_test, nb_classes=2)\n",
    "\n",
    "# Building a LSTM network\n",
    "RNN = tflearn.input_data([None, 100])\n",
    "RNN = tflearn.embedding(RNN, input_dim=10000, output_dim=128)\n",
    "\n",
    "RNN = tflearn.lstm(RNN, 128, dropout=0.8)\n",
    "RNN = tflearn.fully_connected(RNN, 2, activation='softmax')\n",
    "RNN = tflearn.regression(RNN, optimizer='adam', learning_rate=0.001,\n",
    "                         loss='categorical_crossentropy')\n",
    "\n",
    "# Training the network\n",
    "model = tflearn.DNN(RNN, tensorboard_verbose=0)\n",
    "model.fit(X_train, Y_train, validation_set=(X_test, Y_test),\n",
    "                                show_metric=True, batch_size=32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keras\n",
    "### Autoencoding for denoising CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D\n",
    "from keras.models import Model\n",
    "from keras.callbacks import TensorBoard, ModelCheckpoint\n",
    "from keras.datasets import cifar10\n",
    "import numpy as np\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train = x_train[np.where(y_train==1)[0],:,:,:]\n",
    "x_test = x_test[np.where(y_test==1)[0],:,:,:]\n",
    "\n",
    "x_train = x_train.astype('float32') / 255.\n",
    "x_test = x_test.astype('float32') / 255.\n",
    "\n",
    "x_train_n = x_train + 0.5 *\\\n",
    " np.random.normal(loc=0.0, scale=0.4, size=x_train.shape) \n",
    "\n",
    "x_test_n = x_test + 0.5 *\\\n",
    " np.random.normal(loc=0.0, scale=0.4, size=x_test.shape) \n",
    "\n",
    "x_train_n = np.clip(x_train_n, 0., 1.)\n",
    "x_test_n = np.clip(x_test_n, 0., 1.)\n",
    "\n",
    "inp_img = Input(shape=(32, 32, 3))   \n",
    "\n",
    "img= Conv2D(32, (3, 3), activation='relu', padding='same')(inp_img)\n",
    "img = MaxPooling2D((2, 2), padding='same')(img)\n",
    "img = Conv2D(32, (3, 3), activation='relu', padding='same')(img)\n",
    "img = UpSampling2D((2, 2))(img)\n",
    "decoded = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(img)\n",
    "\n",
    "autoencoder = Model(inp_img, decoded)\n",
    "\n",
    "autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy')\n",
    "\n",
    "tensorboard = TensorBoard(log_dir='./models/autoencoder',\\\n",
    "              histogram_freq=0, write_graph=True, write_images=True)\n",
    "model_saver = ModelCheckpoint(\n",
    "                    filepath='./models/autoencoder/autoencoder_model',\\\n",
    "                     verbose=0, period=2)\n",
    "\n",
    "autoencoder.fit(x_train_n, x_train,\n",
    "                epochs=10,\n",
    "                batch_size=64,\n",
    "                shuffle=True,\n",
    "                validation_data=(x_test_n, x_test),\n",
    "                callbacks=[tensorboard, model_saver])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "\n",
    "n_imgs = 10\n",
    "f,axarr = plt.subplots(2,n_imgs,figsize=[20,5])\n",
    "decoded_imgs = autoencoder.predict(x_test_n)\n",
    "\n",
    "for i in range(n_imgs):\n",
    "    \n",
    "    ax = axarr[0,i]\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    ax.imshow(x_test_n[i,:,:,:])\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    \n",
    "    ax = axarr[1,i]\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    ax.imshow(decoded_imgs[i,:,:,:])\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "            \n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Load trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "x_train = x_train[np.where(y_train==1)[0],:,:,:]\n",
    "x_test = x_test[np.where(y_test==1)[0],:,:,:]\n",
    "\n",
    "x_train = x_train.astype('float32') / 255.\n",
    "x_test = x_test.astype('float32') / 255.\n",
    "\n",
    "x_train_n = x_train + 0.5 *\\\n",
    " np.random.normal(loc=0.0, scale=0.4, size=x_train.shape) \n",
    "\n",
    "x_test_n = x_test + 0.5 *\\\n",
    " np.random.normal(loc=0.0, scale=0.4, size=x_test.shape) \n",
    "\n",
    "x_train_n = np.clip(x_train_n, 0., 1.)\n",
    "x_test_n = np.clip(x_test_n, 0., 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D\n",
    "from keras.models import Model\n",
    "from keras.callbacks import TensorBoard, ModelCheckpoint\n",
    "from keras.datasets import cifar10\n",
    "import numpy as np\n",
    "\n",
    "inp_img = Input(shape=(32, 32, 3)) \n",
    "img= Conv2D(32, (3, 3), activation='relu', padding='same')(inp_img)\n",
    "img = MaxPooling2D((2, 2), padding='same')(img)\n",
    "img = Conv2D(32, (3, 3), activation='relu', padding='same')(img)\n",
    "img = UpSampling2D((2, 2))(img)\n",
    "decoded = Conv2D(3, (3, 3), activation='sigmoid', padding='same')(img)\n",
    "\n",
    "autoencoder = Model(inp_img, decoded)\n",
    "Model.load_weights(autoencoder,'./models/autoencoder/autoencoder_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "\n",
    "n_imgs = 10\n",
    "f,axarr = plt.subplots(2,n_imgs,figsize=[20,5])\n",
    "decoded_imgs = autoencoder.predict(x_test_n)\n",
    "\n",
    "for i in range(n_imgs):\n",
    "    \n",
    "    ax = axarr[0,i]\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    ax.imshow(x_test_n[i,:,:,:])\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    \n",
    "    ax = axarr[1,i]\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "    ax.imshow(decoded_imgs[i,:,:,:])\n",
    "    plt.gray()\n",
    "    ax.get_xaxis().set_visible(False)\n",
    "    ax.get_yaxis().set_visible(False)\n",
    "            \n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-Slim - Pretrained VGG16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib import slim\n",
    "sys.path.append(\"/home/itay/git/models/slim\")\n",
    "\n",
    "import sys\n",
    "from datasets import dataset_utils\n",
    "import tensorflow as tf\n",
    "import urllib2\n",
    "from nets import vgg\n",
    "from preprocessing import vgg_preprocessing\n",
    "import os\n",
    "\n",
    "target_dir = '/home/itay/git/checkpoints'\n",
    "\n",
    "url = (\"http://54.68.5.226/car.jpg\")\n",
    "\n",
    "im_as_string = urllib2.urlopen(url).read()  \n",
    "im = tf.image.decode_jpeg(im_as_string, channels=3)\n",
    "\n",
    "image_size = vgg.vgg_16.default_image_size\n",
    "\n",
    "processed_im = vgg_preprocessing.preprocess_image(im,\n",
    "                                                         image_size,\n",
    "                                                         image_size,\n",
    "                                                         is_training=False)\n",
    "\n",
    "processed_images  = tf.expand_dims(processed_im, 0)\n",
    "\n",
    "with slim.arg_scope(vgg.vgg_arg_scope()):\n",
    "     logits, _ = vgg.vgg_16(processed_images,\n",
    "                            num_classes=1000,\n",
    "                             is_training=False)\n",
    "probabilities = tf.nn.softmax(logits)\n",
    "\n",
    "def vgg_arg_scope(weight_decay=0.0005):\n",
    "    with slim.arg_scope([slim.conv2d, slim.fully_connected],\n",
    "                     activation_fn=tf.nn.relu,\n",
    "                     weights_regularizer=slim.l2_regularizer(weight_decay),\n",
    "                     biases_initializer=tf.zeros_initializer):\n",
    "    with slim.arg_scope([slim.conv2d], padding='SAME') as arg_sc:\n",
    "        return arg_sc\n",
    "\n",
    "\n",
    "\n",
    "load_vars = slim.assign_from_checkpoint_fn(\n",
    "     os.path.join(target_dir, 'vgg_16.ckpt'),\n",
    "     slim.get_model_variables('vgg_16'))\n",
    "\n",
    "\n",
    "from datasets import imagenet\n",
    "imagenet.create_readable_names_for_imagenet_labels()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Infer class and probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "names = []\n",
    "with tf.Session() as sess:\n",
    "    load_vars(sess)     \n",
    "    network_input, probabilities = sess.run([processed_images,\n",
    "                                             probabilities])\n",
    "    probabilities = probabilities[0, 0:]\n",
    "    names_ = imagenet.create_readable_names_for_imagenet_labels()\n",
    "    idxs = np.argsort(-probabilities)[:5]\n",
    "    probs = probabilities[idxs]\n",
    "    classes = np.array(names_.values())[idxs+1]\n",
    "    for c,p in zip(classes,probs):\n",
    "        print('Class: '+ c + ' |Prob: ' + str(p))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf_1_0",
   "language": "python",
   "name": "tf_1_0"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}